/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package LM;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.StringTokenizer;

/**
 *
 * @author Administrator
 */
public class LanguageModel {

    private HashMap<String, Integer> data = new HashMap<String, Integer>();
    private int total = 0;

//    public static void main(String argv[]) throws FileNotFoundException, UnsupportedEncodingException, IOException {
//        LanguageModel lm = new LanguageModel();
//        lm.createModel("data\\llm\\awsd.db");
//        lm.writeModelFile("data\\llm\\awsdLlm.db");
//    }
    public HashMap<String, Integer> getData() {
        return data;
    }

    public int getTotal() {
        return total;
    }

    public ArrayList readInput() throws FileNotFoundException, UnsupportedEncodingException, IOException {
        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream("data\\lm\\lmdata.db"), "UTF8"));
        ArrayList inputData = new ArrayList();
        String line;
        while ((line = br.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line, "|");
            ArrayList doc = new ArrayList();
            while (st.hasMoreTokens()) {
                doc.add(st.nextToken());
            }
            inputData.add(doc);
        }
        return inputData;
    }

    public void createModel() throws FileNotFoundException, UnsupportedEncodingException, IOException {

        System.out.println("package LM; class LanguageModel");

        //get inputData
        ArrayList inputData = readInput();
        //set data
        for (int i = 0; i < inputData.size(); i++) {
            ArrayList doc = (ArrayList) inputData.get(i);
            for (int j = 0; j < doc.size(); j++) {
                String wordGroup = "";
                if (j == 0) {
                    wordGroup = (String) doc.get(j);
                } else {
                    if (j == 1) {
                        wordGroup = (String) doc.get(j - 1) + "|" + (String) doc.get(j);
                    } else {
                        wordGroup = (String) doc.get(j - 2) + "|" + (String) doc.get(j - 1) + "|" + (String) doc.get(j);
                    }

                }
                if (data.containsKey(wordGroup)) {
                    int count = data.get(wordGroup);
                    count++;
                    data.put(wordGroup, count);
                } else {
                    data.put(wordGroup, 1);
                }
            }
        }
        //set total
        for (Map.Entry<String, Integer> e : data.entrySet()) {
            total += e.getValue();
        }
    }

    public double getEntropy(ArrayList doc) {
        double entropy = 1;
        for (int j = 0; j < doc.size(); j++) {
            String wordGroup = "";
            if (j == 0) {
                wordGroup = (String) doc.get(j);
            } else {
                if (j == 1) {
                    wordGroup = (String) doc.get(j - 1) + "|" + (String) doc.get(j);
                } else {
                    wordGroup = (String) doc.get(j - 2) + "|" + (String) doc.get(j - 1) + "|" + (String) doc.get(j);
                }

            }
            if (data.containsKey(wordGroup)) {
                entropy = entropy * data.get(wordGroup) / total;
            } else {
                entropy = entropy * 0.000000000001;
            }

        }
        return entropy;
    }

    public void writeModelFile() throws FileNotFoundException, UnsupportedEncodingException, IOException {
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream("data\\lm\\lm.db"), "UTF8"));

        ArrayList dataList = new ArrayList(data.entrySet());
        Collections.sort(dataList, new Comparator() {

            public int compare(Object o1, Object o2) {
                Map.Entry e1 = (Map.Entry) o1;
                Map.Entry e2 = (Map.Entry) o2;
                Integer first = (Integer) e1.getValue();
                Integer second = (Integer) e2.getValue();
                return second.compareTo(first);
            }
        });
        for (int i = 0; i < dataList.size(); i++) {
            Map.Entry e = (Map.Entry) dataList.get(i);
            String line = e.getKey() + "_" + e.getValue() + "\n";
            bw.write(line);
        }
        bw.close();

    }

    public void loadModel(String filename) throws FileNotFoundException, UnsupportedEncodingException, IOException {
        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "UTF8"));
        total = 0;
        String line;
        while ((line = br.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line, "_");
            String str = st.nextToken();
            int num = Integer.parseInt(st.nextToken());
            data.put(str, num);
            total += num;
        }

    }
}
